#!usr/bin/python3
#-*- coding:utf-8 -*-

import asyncio
import requests
import pymysql
import re,os,stat
import time
from requests.adapters import HTTPAdapter
from urllib.request import urlretrieve
from lxml import etree
from db import Db

host = 'http://www.xiachufang.com'
dbC = Db()
db = dbC.dbConfig()
cursor = db.cursor(cursor = pymysql.cursors.DictCursor)

async def main(url,cat_id):
    # print(1)

    responseDict = await asyncio.ensure_future(getContent(url))
    query_data = selectData(responseDict['text'],url)
    query_data_total = []
    for index,data in enumerate(query_data):
        foodDetailUrl = '{0}/{1}'.format(host,data['foodDetailUrl'])
        detailDict = await asyncio.ensure_future(getContent(foodDetailUrl,index))
        data = selectDetailData(detailDict['text'],query_data[detailDict['index']])
        await asyncio.ensure_future(insertDb(data,cat_id))
        
    
        
async def getContent(url,index = None):
    cookies_dict = {}
    cookies = ''
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36',
        'Accept':'text/html',
        'Accept-Encoding':'gzip, deflate, br',
        'Accept-Language':'zh-CN,zh;q=0.9,en;q=0.8'
        # 'Referer':'http://www.sduod.com/baike/anli/'
    }

    cookies_arr = cookies.split(';')
        
    for one_c in cookies_arr:
        try:
            one_arr = one_c.split('=')
            k = one_arr[0]
            v = one_arr[1]
            cookies_dict[k] = v
        except Exception as e:
            pass

    s = requests.Session()
    s.mount('http://',HTTPAdapter(max_retries=2))
    s.mount('https://',HTTPAdapter(max_retries=2))
    try:
        response = s.get(url,cookies=cookies_dict,headers=headers,timeout=10)
        response.encoding = 'utf-8'
    except Exception as e:
        print(e)
        response = {
            'text':''
        }
    
    return {'index':index,'text':response.text}

def selectData(content,url):
    return_list = []
    if content:
        html = etree.HTML(content)
        try:
            foodList = html.xpath('//div[@class="dishes-280-full-width-list"]/ul[1]/li')
            if not foodList:
                return return_list
            for food in foodList:
                return_dict = {}
                try:
                    foodName = food.xpath('./child::div[1]/p[1]/a[1]/text()')[0]
                    foodDetailUrl = food.xpath('./child::div[1]/p[1]/a[1]/@href')[0]
                    foodDesc = food.xpath('./child::div[1]/p[2]/text()')[0]
                    foodGood = food.xpath('./child::div[1]/div[2]/p[2]/span[1]/text()')[0]
                    foodCover = food.xpath('./child::div[1]/div[1]/img/@data-src')[0]
                    memberName = food.xpath('./child::div[1]/div[3]/a[1]/@title')[0]
                    memberImg = food.xpath('./child::div[1]/div[3]/a[1]/img/@data-src')[0]
                    memberAddress_arr = food.xpath('./child::div[1]/div[3]/span[1]/text()')

                    if memberAddress_arr:
                        memberAddress = memberAddress_arr[0]
                        addrReg = re.compile(r'（(.*?)）',re.S)
                        memberAddress = re.search(addrReg,memberAddress).group(1)
                    else:
                        memberAddress = ''
                          
                    

                    return_dict['foodName'] = foodName
                    return_dict['foodDetailUrl'] = foodDetailUrl
                    return_dict['foodDesc'] = foodDesc
                    return_dict['foodGood'] = foodGood
                    return_dict['foodCover'] = foodCover
                    return_dict['memberName'] = memberName
                    return_dict['memberImg'] = memberImg
                    return_dict['memberAddress'] = memberAddress
                    return_list.append(return_dict)
                except Exception as e:
                    # print(e)
                    print('获取食物信息错误:'+foodName)
                    continue
        except Exception as e:
            print('获取食物列表失败'+url)
    return return_list

#获取详情页信息
def selectDetailData(content,data):
    if content:
        dHtml = etree.HTML(content)
        try:
            foodPublicTimeEle = dHtml.xpath('//div[@class="pure-u dish-user-info"]/div[3]')[0]
            foodPublicTimeEle = etree.tostring(foodPublicTimeEle,encoding='utf-8').decode('utf-8')
                                    
            timeReg = re.compile(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}')
            foodPublicTime = re.search(timeReg,foodPublicTimeEle).group(0)
            data['foodPublicTime'] = foodPublicTime                                         
        except Exception as e:
            data['foodPublicTime'] = ''
            print('食物详情页获取失败:'+data['foodDetailUrl'])
    return data

async def insertDb(data,cat_id):
    #先入库再下载图片
    if data:
        foodCover = data['foodCover']
        imgPath = './images/cover'
        coverUrl = handleImg(foodCover,imgPath)
        memberImg = data['memberImg']
        imgPath = './images/head'
        memberImgUrl = handleImg(memberImg,imgPath)

        #先插入会员 （先查询是否存在）
        memberData = dict()
        try:
            sql = "select * from member where name = %s"
            name = (data['memberName'])
            cursor.execute(sql,name)
            memberData = cursor.fetchone()
        except Exception as e:
            print('查询用户失败')
            return False
        if not memberData:
            try: 
                sql = "insert ignore into member(name,address,img) values(%s,%s,%s)"
                insertData = (name,data['memberAddress'],memberImgUrl)
                res = cursor.execute(sql,insertData)
                member_id = cursor.lastrowid
                if not member_id:
                    return False
            except Exception as e:
                print('插入用户失败')
        else:
            member_id = memberData['id'] 
        db.commit()
        #插入食物数据（先查询食物是否存在）
        food_data = dict()
        try:
            sql = "select id from food where title = %s and member_id = %s and createtime = %s"
            food_tup = (data['foodName'],member_id,data['foodPublicTime'])
            cursor.execute(sql,food_tup)
            food_data = cursor.fetchone()
        except Exception as e:
            print('食物查询失败')
            # exit()
            return False
        if not food_data:
            try:
                sql = "insert ignore into food(title,food_desc,cover,good,member_id,createtime) values(%s,%s,%s,%s,%s,%s)"
                food_insert_data = (data['foodName'],data['foodDesc'],coverUrl,data['foodGood'],member_id,data['foodPublicTime'])
                res = cursor.execute(sql,food_insert_data)
                food_id = cursor.lastrowid
                if not food_id:
                    return False
            except Exception as e:
                print('食物插入失败')
                return False

        else:
            food_id = food_data['id']

        #插入食物与分类的对应关系（先查询对应关系是否存在）
        candf_data = dict()
        try:
            sql = "select id from candf where food_id = %s and cat_id = %s"
            candf_tup = (food_id,cat_id)
            cursor.execute(sql,candf_tup) 
            candf_data = cursor.fetchone()
        except Exception as e:
            print('查询食物与分类关系错误')
            db.rollback()
            return False
        if not candf_data:
            try:
                sql = "insert ignore into candf(food_id,cat_id) values(%s,%s)"
                candf_tup = (food_id,cat_id)
                cursor.execute(sql,candf_tup)
            except Exception as e:
                print('插入食物与分类错误')
                db.rollback()
                return False
        db.commit()
        downloadPic(foodCover,coverUrl)
        downloadPic(memberImg,memberImgUrl)


def handleImg(imgUrl,dirpath):
    if imgUrl.find('?') >= 0:
        foodCover_arr = imgUrl.split('?')
        coverName = os.path.basename(foodCover_arr[0])
        # coverName = '{0}?{1}'.format(coverName,foodCover_arr[1])
    elif imgUrl.find('@') >= 0:
        foodCover_arr = imgUrl.split('@')
        coverName = os.path.basename(foodCover_arr[0])
    else:
        coverName = os.path.basename(imgUrl)
    if not os.path.exists(dirpath):
        dirpath = createDir(dirpath)
    if dirpath:
        coverUrl = '{0}/{1}'.format(dirpath,coverName)
    else:
        coverUrl = ''
    return coverUrl

def createDir(paths):
    if paths:
        try:
            os.makedirs(paths)
            os.chmod(paths,stat.S_IRWXU|stat.S_IRWXG|stat.S_IROTH)
        except Exception as e:
            paths = ''
            print('下载封面失败')
    else:
        paths = ''
    return paths

    

def downloadPic(url,imgpath):
    if imgpath:
        try:
            if not os.path.exists(imgpath):
                urlretrieve(url,imgpath)
        except Exception as e:
            print('下载失败')

        
#获取分类列表
def getCatList():
    try:
        sql = 'select * from category'
        cursor.execute(sql)
        cats = cursor.fetchall()

        if not cats:
            cats = []

    except Exception as e:
        cats = []
    return cats
def start():
    start_time = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
    print('----采集开始 时间：{0}----'.format(start_time))
    cats = getCatList()
    loop = asyncio.get_event_loop()
    tasks = [asyncio.ensure_future(main(one['url'],one['id'])) for one in cats]
    loop.run_until_complete(asyncio.wait(tasks))
    end_time = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
    print('----采集完成 时间：{0}----'.format(end_time))
    print('----休眠60s----\n')
    time.sleep(60)   
    start() 

if __name__ == '__main__':
    start()
    # print(responseText)

    




    

